package com.gylang.crawler.task;

import com.gylang.crawler.crawler.EsLogDownloader;
import com.gylang.crawler.crawler.dy2018.Dy2018CategoryProcessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;

/**
 * @author gylang
 * data 2021/5/28
 */
@Component
public class Dy2018CrawlerTask {


    @Bean
    public Spider dy2018Spider(@Autowired Dy2018CategoryProcessor categoryProcessor,
                               @Autowired EsLogDownloader esLogDownloader) {
        Spider spider = Spider.create(categoryProcessor)
                .addUrl("https://www.dy2018.com/1/")
                .setDownloader(esLogDownloader)
                .thread(3);
        spider.runAsync();
        return spider;
    }
}
